package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/http"
	"os"
)

func main() {
	//创建导出文件
	file, err := os.Create("测试爬虫.txt")
	if err != nil {
		fmt.Println(err.Error())
	}
	defer file.Close()
	//创建客户端对象，发送请求
	var client = http.Client{}
	for i := 0; i < 250; i += 250 {
		//发送请求
		req, _ := http.NewRequest("GET", "https://www.xbiquge.la/10/10489/4535761.html", nil)
		//req.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36")
		//发送请求
		// 发送请求
		resp, err := client.Do(req)
		// 处理异常
		if err != nil {
			fmt.Println("http get error", err)
			return
		}
		// 关闭流
		defer resp.Body.Close()
		doc, err := goquery.NewDocumentFromReader(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		chapterNmae := doc.Find("h1").Text()
		fmt.Println(chapterNmae)
		file.WriteString(chapterNmae+"\n")
		contentTxt := doc.Find("#content").Text()
		fmt.Println(contentTxt)
		file.WriteString(contentTxt)
	}
	fmt.Print("程序执行完毕，请查看结果。")
}